/*
 * Indexer for the dictionary with different fields
 *
 * ZhihuiJin <zhihuijin@gmail.com>
 * vim: ts=4 sw=4 sts=4 et tw=78:
 * Thu Feb  8 13:28:03 JST 2007
 */ 

import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import java.io.*;
import java.util.*;


public class DictIndexer{

	//fields analyzer, do word segmentation in whitespace
	static Analyzer analyzer= new WhitespaceAnalyzer();		
	
	/** build index for the dictionary data
	 *
	 * @param indexDir the directory to store index data
	 * @param dataDir the directory of dictionary data
	 * @throw IOException if dataDir doesn't exit
	 */
	public static void index(File indexDir, File dataDir) throws IOException {
		if (!dataDir.exists() || !dataDir.isDirectory()) {
			throw new IOException(dataDir + " does not exist or is not a directory");
		}

		IndexWriter writer = new IndexWriter(indexDir, analyzer, true);
		indexDirectory(writer, dataDir);
		writer.optimize();
		writer.close();
	}


	/** scan the directonry of dictionary data, indexing each file
	 *
	 * @param writer index writer
	 * @param dir directory of dictionary data
	 */
	private static void indexDirectory(IndexWriter writer, File dir) throws IOException {
		File[] files = dir.listFiles();

		for (int i=0; i < files.length; i++) {
			File f = files[i];
			if (f.isDirectory()) {
				indexDirectory(writer, f);  // recurse
			} else if (f.getName().endsWith(".txt")||f.getName().endsWith(".TXT")) { //corpus file must be *.txt
				indexFile(writer, f);
			}
		}
	}


	/** Index each single dictionary data file
	 * @param writer index writer
	 * @param f data file
	 */
	private static void indexFile(IndexWriter writer, File f) throws IOException {
		System.out.println("Indexing " + f.getName());

		BufferedReader in = new BufferedReader(new FileReader(f));
		Vector fieldVec = new Vector();
		String strLine  = null;	
		String fieldSep = ":";
		int docCount    = 0;

		while((strLine = in.readLine()) != null){
			if (!strLine.trim().equals("")) {
				fieldVec.addElement(strLine);
				continue;
			}

			Document doc = new Document();		
			for (int i = 0; i < fieldVec.size(); i++) { //add fields into document
				String   fieldStr  = (String) fieldVec.get(i);
				String[] fieldInfo = fieldStr.split(fieldSep, 3);

				switch (Integer.parseInt(fieldInfo[0])) {	//check field type
					case 0:
						doc.add(new Field(fieldInfo[1], fieldInfo[2], Field.Store.NO, Field.Index.TOKENIZED));
						break;
					case 1:
						doc.add(new Field(fieldInfo[1], fieldInfo[2], Field.Store.YES, Field.Index.UN_TOKENIZED));
						break;
					default:
						throw new IOException("Wrong field type in : " + fieldStr);
				}
			}

			writer.addDocument(doc);
			docCount++;
			fieldVec.clear();

			if (docCount % 5000 == 0)
				System.out.println("...... " + Integer.toString(docCount) + " documents indexed");
		}

		System.out.println("Totally " + Integer.toString(docCount) + " documents indexed");
		System.out.println();
	}

   public static void main(String[] args) throws Exception {
        if (args.length != 2) {
            throw new Exception("\nUsage: DictIndexer"  + " <index dir> <data dir>\n");
        }
        File indexDir = new File(args[0]);
        File dataDir = new File(args[1]);
        index(indexDir, dataDir);
    }

}


